# -*- coding: utf-8 -*-
import scrapy
from pandas import  json
import re
class ElongSpider(scrapy.Spider):
    name = 'Elong'
    allowed_domains = ['elong.com']
    start_urls = ['http://hotel.elong.com/']
    count = 0
    cities = []
    def parse(self, response):
        with open('cities.txt', 'r') as file:
            lines = file.readlines()
            for line in lines:
                city = line.split()[1]
                self.cities.append(city.lower())
        hotel_href = self.getGroup(response)
        groupPage = self.getGroupPage(hotel_href)
        for gp in groupPage:
            yield gp
        hotelIdBegin = '00201020'
        hotelIdEnd = '00201425'
        forward = 'http://hotel.elong.com/ajax/comment/getcommentbypage/?recommendedType=0&pageIndex=0&mainTagId=0&subTagId=0&rankType=0&eToken=1721e10b-143d-4825-a764-8df95852bba0&code=9159708&_=1551196470689'
        for i in range(int(hotelIdBegin), int(hotelIdEnd)):
            href = forward + '&hotelId=00' + str(i)
            yield scrapy.Request('http://hotel.elong.com/00' + str(i) + '/',callback=self.parsePage)
            yield scrapy.Request(href, meta={'commentCount': 0}, callback= self.parseComment)
    def parseSubHotel(self, response):
        sub_hotel_css = 'body > div.grid > div:nth-child(5) > div > ul > li'
        sub_hotel_href = response.selector.css(sub_hotel_css)
        if (sub_hotel_href.get() is None):
            return
        else:
            sub_hotel_xpath = './/a/@href'
            hrefs = sub_hotel_href.xpath(sub_hotel_xpath).getall()
            for href in hrefs:
                city = href.split('/')[-3]
                if (city in self.cities):
                    href = href.replace(city+'/','')
                yield scrapy.Request(href, callback=self.parsePage)
                request = self.getComment(href)
                for r in request:
                    yield r
    def getGroup(self, response):
        hotel_band_css = 'body > div.hi_content > div.pub_wrap.recom_brand > div.mt20.recom_brand_b > ul > li'
        hotel_name_xpath = './/a/@title'
        hotel_href_xpath = './/a/@href'
        hotel_href_html = response.selector.css(hotel_band_css)
        if (hotel_href_html.get() is None):
            return {}
        hotel_name = hotel_href_html.xpath(hotel_name_xpath).getall()
        hotel_href = hotel_href_html.xpath(hotel_href_xpath).getall()
        answer = {}
        for index in range(len(hotel_name)):
            if (hotel_href[index].find('html') != -1):
                answer.setdefault(hotel_name[index], hotel_href[index])
        return answer

    def getGroupPage(self, dicts):
        for key in dicts:
            yield scrapy.Request(dicts[key], callback=self.parseSubHotel, dont_filter=True)

    def parsePage(self, response):
        tag_css = '#review > div.cmt_hd > div.cmt_tp > div.cmt_sylst > div > a'
        tags = response.selector.css(tag_css)
        tags = tags.xpath('.//text()').extract()
        with open('Elong_TAG.csv', 'a', encoding='utf8') as file:
            for tag in tags:
                file.write(tag)

    def getComment(self, url):
        hotel = 'hotelId='
        forward = 'http://hotel.elong.com/ajax/comment/getcommentbypage/?recommendedType=0&pageIndex=0&mainTagId=0&subTagId=0&rankType=0&eToken=1721e10b-143d-4825-a764-8df95852bba0&code=9159708&_=1551196470689'
        hotelId = url.split('/')[-2]
        href = forward + '&' + hotel + str(hotelId)
        yield scrapy.Request(href, meta={'commentCount': 0}, callback=self.parseComment)

    def parseComment(self, response):
        originalCount = response.meta['commentCount']
        response_content = response.text
        response_content = json.loads(response_content)
        values = response_content['value']
        if ('Comments' in values.keys()):
            comments = values['Comments']
            originalCount = originalCount + len(comments)
            if (originalCount < int(values['TotalNumber'])):
                print('total number: %d, current number: %d' % (int(values['TotalNumber']), originalCount))
                current_page = re.search(r'pageIndex=[0-9]{1,6}',response.url)
                if (current_page):
                    current_page = current_page.group(0)
                    key, page = current_page.split('=')
                    page = int(page)
                new_url = re.sub(key+'='+str(page), key+'='+str(page+1), response.url)
                yield scrapy.Request(new_url, meta={'commentCount': originalCount}, callback=self.parseComment)
            with open('Elong_comment.csv', 'a', encoding='utf8') as file:
                for comment in comments:
                    file.write(comment['Content'] + '\n')

        else:
            return